{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Description:\n",
    "这个jupyter是建立Deep Crossing模型并完成训练， 这个模型结构相对来说比较简单， 我们可以看一下结构：\n",
    "\n",
    "![](img/1.png)\n",
    "下面尝试用Pytorch建立这个模型， 并使用Pytorch完成模型的训练和测试， 一方面对这个结构有个更深的认识， 另一方面也可以把Pytorch的知识应用起来。关于Pytorch的建模流程， 主要有四步：\n",
    "1. 准备数据\n",
    "2. 建立模型\n",
    "3. 训练模型\n",
    "4. 使用和保存\n",
    "\n",
    "基于之前学习的Pytorch知识， 利用这个机会把这些都串一下。 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入包和数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:27.995998Z",
     "start_time": "2020-10-07T02:15:26.539313Z"
    }
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch \n",
    "from torch.utils.data import TensorDataset, Dataset, DataLoader\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchkeras import summary, Model\n",
    "\n",
    "from sklearn.metrics import auc, roc_auc_score, roc_curve\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:28.026576Z",
     "start_time": "2020-10-07T02:15:27.998611Z"
    }
   },
   "outputs": [],
   "source": [
    "# 导入数据， 数据已经处理好了 preprocess/下\n",
    "train_set = pd.read_csv('preprocessed_data/train_set.csv')\n",
    "val_set = pd.read_csv('preprocessed_data/val_set.csv')\n",
    "test_set = pd.read_csv('preprocessed_data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:28.058448Z",
     "start_time": "2020-10-07T02:15:28.027532Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>I1</th>\n",
       "      <th>I2</th>\n",
       "      <th>I3</th>\n",
       "      <th>I4</th>\n",
       "      <th>I5</th>\n",
       "      <th>I6</th>\n",
       "      <th>I7</th>\n",
       "      <th>I8</th>\n",
       "      <th>I9</th>\n",
       "      <th>I10</th>\n",
       "      <th>...</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "      <th>C22</th>\n",
       "      <th>C23</th>\n",
       "      <th>C24</th>\n",
       "      <th>C25</th>\n",
       "      <th>C26</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000127</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.017115</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>463</td>\n",
       "      <td>23</td>\n",
       "      <td>2</td>\n",
       "      <td>554</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>539</td>\n",
       "      <td>27</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000127</td>\n",
       "      <td>0.002010</td>\n",
       "      <td>0.045977</td>\n",
       "      <td>0.000608</td>\n",
       "      <td>0.012290</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.007313</td>\n",
       "      <td>0.000710</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>263</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>407</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>635</td>\n",
       "      <td>1</td>\n",
       "      <td>206</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.031579</td>\n",
       "      <td>0.016145</td>\n",
       "      <td>0.001182</td>\n",
       "      <td>0.114943</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.001078</td>\n",
       "      <td>0.008444</td>\n",
       "      <td>0.010969</td>\n",
       "      <td>0.002129</td>\n",
       "      <td>0.25</td>\n",
       "      <td>...</td>\n",
       "      <td>188</td>\n",
       "      <td>115</td>\n",
       "      <td>3</td>\n",
       "      <td>971</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>567</td>\n",
       "      <td>27</td>\n",
       "      <td>465</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021485</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000195</td>\n",
       "      <td>0.052178</td>\n",
       "      <td>0.003619</td>\n",
       "      <td>0.001828</td>\n",
       "      <td>0.001597</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>387</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>163</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000127</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>233</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>66</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>44</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         I1        I2        I3        I4        I5        I6        I7  \\\n",
       "0  0.000000  0.000127  0.000000  0.000000  0.017115  0.000000  0.000000   \n",
       "1  0.000000  0.000127  0.002010  0.045977  0.000608  0.012290  0.000000   \n",
       "2  0.031579  0.016145  0.001182  0.114943  0.000002  0.001078  0.008444   \n",
       "3  0.000000  0.021485  0.000000  0.000000  0.000195  0.052178  0.003619   \n",
       "4  0.000000  0.000127  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "\n",
       "         I8        I9   I10  ...  C18  C19  C20  C21  C22  C23  C24  C25  C26  \\\n",
       "0  0.000000  0.000000  0.00  ...  463   23    2  554    0    0  539   27   49   \n",
       "1  0.007313  0.000710  0.00  ...  263   20    2  407    0    1  635    1  206   \n",
       "2  0.010969  0.002129  0.25  ...  188  115    3  971    0    7  567   27  465   \n",
       "3  0.001828  0.001597  0.00  ...  387    0    0    2    0    3  163    0    0   \n",
       "4  0.000000  0.000000  0.00  ...  233    0    0   66    0    0   44    0    0   \n",
       "\n",
       "   Label  \n",
       "0      0  \n",
       "1      0  \n",
       "2      1  \n",
       "3      0  \n",
       "4      0  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:28.073451Z",
     "start_time": "2020-10-07T02:15:28.060443Z"
    },
    "run_control": {
     "marked": true
    }
   },
   "outputs": [],
   "source": [
    "# 这里需要把特征分成数值型和离散型， 因为后面的模型里面离散型的特征需要embedding， 而数值型的特征直接进入了stacking层， 处理方式会不一样\n",
    "data_df = pd.concat((train_set, val_set, test_set))\n",
    "\n",
    "dense_feas = ['I'+str(i) for i in range(1, 14)]\n",
    "sparse_feas = ['C'+str(i) for i in range(1, 27)]\n",
    "\n",
    "# 定义一个稀疏特征的embedding映射， 字典{key: value}, key表示每个稀疏特征， value表示数据集data_df对应列的不同取值个数， 作为embedding输入维度\n",
    "sparse_feas_map = {}\n",
    "for key in sparse_feas:\n",
    "    sparse_feas_map[key] = data_df[key].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:28.089408Z",
     "start_time": "2020-10-07T02:15:28.074406Z"
    }
   },
   "outputs": [],
   "source": [
    "feature_info = [dense_feas, sparse_feas, sparse_feas_map]  # 这里把特征信息进行封装， 建立模型的时候作为参数传入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:28.104397Z",
     "start_time": "2020-10-07T02:15:28.090363Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['I1', 'I2', 'I3', 'I4', 'I5', 'I6', 'I7', 'I8', 'I9', 'I10', 'I11',\n",
       "       'I12', 'I13', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9',\n",
       "       'C10', 'C11', 'C12', 'C13', 'C14', 'C15', 'C16', 'C17', 'C18', 'C19',\n",
       "       'C20', 'C21', 'C22', 'C23', 'C24', 'C25', 'C26'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.columns\n",
    "test_set.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:28.120326Z",
     "start_time": "2020-10-07T02:15:28.106320Z"
    }
   },
   "outputs": [],
   "source": [
    "# 把数据构建成数据管道\n",
    "dl_train_dataset = TensorDataset(torch.tensor(train_set.drop(columns='Label').values).float(), torch.tensor(train_set['Label'].values).float())\n",
    "dl_val_dataset = TensorDataset(torch.tensor(val_set.drop(columns='Label').values).float(), torch.tensor(val_set['Label'].values).float())\n",
    "\n",
    "dl_train = DataLoader(dl_train_dataset, shuffle=True, batch_size=16)\n",
    "dl_vaild = DataLoader(dl_val_dataset, shuffle=True, batch_size=16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 建立模型\n",
    "常用的三种方式\n",
    "1. 继承nn.Module基类构建自定义模型\n",
    "2. nn.Sequential按层顺序构建模型\n",
    "3. 继承nn.Module基类构建模型， 并辅助应用模型容器进行封装\n",
    "\n",
    "这里我们使用第三种方式搭建DeepCrossing模型， 因为这里的embedding层需要多种\n",
    "\n",
    "![](img/1.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:28.135244Z",
     "start_time": "2020-10-07T02:15:28.123276Z"
    }
   },
   "outputs": [],
   "source": [
    "# 首先， 自定义一个残差块\n",
    "class Residual_block(nn.Module):\n",
    "    \"\"\"\n",
    "    Define Residual_block\n",
    "    \"\"\"\n",
    "    def __init__(self, hidden_unit, dim_stack):\n",
    "        super(Residual_block, self).__init__()\n",
    "        self.linear1 = nn.Linear(dim_stack, hidden_unit)\n",
    "        self.linear2 = nn.Linear(hidden_unit, dim_stack)\n",
    "        self.relu = nn.ReLU()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        orig_x = x.clone()\n",
    "        x = self.linear1(x)\n",
    "        x = self.linear2(x)\n",
    "        outputs = self.relu(x + orig_x)\n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:28.151202Z",
     "start_time": "2020-10-07T02:15:28.137238Z"
    }
   },
   "outputs": [],
   "source": [
    "# 定义deep Crossing 网络\n",
    "class DeepCrossing(nn.Module):\n",
    "    \n",
    "    def __init__(self, feature_info, hidden_units, dropout=0., embed_dim=10, output_dim=1):\n",
    "        \"\"\"\n",
    "        DeepCrossing：\n",
    "            feature_info: 特征信息（数值特征， 类别特征， 类别特征embedding映射)\n",
    "            hidden_units: 列表， 隐藏单元的个数(多层残差那里的)\n",
    "            dropout: Dropout层的失活比例\n",
    "            embed_dim: embedding维度\n",
    "        \"\"\"\n",
    "        super(DeepCrossing, self).__init__()\n",
    "        self.dense_feas, self.sparse_feas, self.sparse_feas_map = feature_info\n",
    "        \n",
    "        # embedding层， 这里需要一个列表的形式， 因为每个类别特征都需要embedding\n",
    "        self.embed_layers = nn.ModuleDict({\n",
    "            'embed_' + str(key): nn.Embedding(num_embeddings=val, embedding_dim=embed_dim)\n",
    "            for key, val in self.sparse_feas_map.items()\n",
    "        })\n",
    "        \n",
    "        # 统计embedding_dim的总维度\n",
    "        embed_dim_sum = sum([embed_dim]*len(self.sparse_feas))\n",
    "        \n",
    "        # stack layers的总维度\n",
    "        dim_stack = len(self.dense_feas) + embed_dim_sum\n",
    "        \n",
    "        # 残差层\n",
    "        self.res_layers = nn.ModuleList([\n",
    "            Residual_block(unit, dim_stack) for unit in hidden_units\n",
    "        ])\n",
    "        \n",
    "        # dropout层\n",
    "        self.res_dropout = nn.Dropout(dropout)\n",
    "        \n",
    "        # 线性层\n",
    "        self.linear = nn.Linear(dim_stack, output_dim)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        dense_inputs, sparse_inputs = x[:, :13], x[:, 13:]\n",
    "        sparse_inputs = sparse_inputs.long()      # 需要转成长张量， 这个是embedding的输入要求格式\n",
    "        sparse_embeds = [self.embed_layers['embed_'+key](sparse_inputs[:, i]) for key, i in zip(self.sparse_feas_map.keys(), range(sparse_inputs.shape[1]))]   \n",
    "        sparse_embed = torch.cat(sparse_embeds, axis=-1)\n",
    "        stack = torch.cat([sparse_embed, dense_inputs], axis=-1)\n",
    "        r = stack\n",
    "        for res in self.res_layers:\n",
    "            r = res(r)\n",
    "        \n",
    "        r = self.res_dropout(r)\n",
    "        outputs = F.sigmoid(self.linear(r))\n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:27:59.993190Z",
     "start_time": "2020-10-07T02:27:59.960277Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "         Embedding-1                   [-1, 10]             790\n",
      "         Embedding-2                   [-1, 10]           2,520\n",
      "         Embedding-3                   [-1, 10]          12,930\n",
      "         Embedding-4                   [-1, 10]          10,430\n",
      "         Embedding-5                   [-1, 10]             300\n",
      "         Embedding-6                   [-1, 10]              70\n",
      "         Embedding-7                   [-1, 10]          11,640\n",
      "         Embedding-8                   [-1, 10]             390\n",
      "         Embedding-9                   [-1, 10]              20\n",
      "        Embedding-10                   [-1, 10]           9,080\n",
      "        Embedding-11                   [-1, 10]           9,260\n",
      "        Embedding-12                   [-1, 10]          12,390\n",
      "        Embedding-13                   [-1, 10]           8,240\n",
      "        Embedding-14                   [-1, 10]             200\n",
      "        Embedding-15                   [-1, 10]           8,190\n",
      "        Embedding-16                   [-1, 10]          11,590\n",
      "        Embedding-17                   [-1, 10]              90\n",
      "        Embedding-18                   [-1, 10]           5,340\n",
      "        Embedding-19                   [-1, 10]           2,010\n",
      "        Embedding-20                   [-1, 10]              40\n",
      "        Embedding-21                   [-1, 10]          12,040\n",
      "        Embedding-22                   [-1, 10]              70\n",
      "        Embedding-23                   [-1, 10]             120\n",
      "        Embedding-24                   [-1, 10]           7,290\n",
      "        Embedding-25                   [-1, 10]             330\n",
      "        Embedding-26                   [-1, 10]           5,540\n",
      "           Linear-27                  [-1, 256]          70,144\n",
      "           Linear-28                  [-1, 273]          70,161\n",
      "             ReLU-29                  [-1, 273]               0\n",
      "           Linear-30                  [-1, 128]          35,072\n",
      "           Linear-31                  [-1, 273]          35,217\n",
      "             ReLU-32                  [-1, 273]               0\n",
      "           Linear-33                   [-1, 64]          17,536\n",
      "           Linear-34                  [-1, 273]          17,745\n",
      "             ReLU-35                  [-1, 273]               0\n",
      "           Linear-36                   [-1, 32]           8,768\n",
      "           Linear-37                  [-1, 273]           9,009\n",
      "             ReLU-38                  [-1, 273]               0\n",
      "          Dropout-39                  [-1, 273]               0\n",
      "           Linear-40                    [-1, 1]             274\n",
      "================================================================\n",
      "Total params: 394,836\n",
      "Trainable params: 394,836\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.000153\n",
      "Forward/backward pass size (MB): 0.024399\n",
      "Params size (MB): 1.506180\n",
      "Estimated Total Size (MB): 1.530731\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "hidden_units = [256, 128, 64, 32]\n",
    "net = DeepCrossing(feature_info, hidden_units)\n",
    "summary(net, input_shape=(train_set.shape[1],))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:28.197078Z",
     "start_time": "2020-10-07T02:15:28.184114Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.6328],\n",
      "        [0.5989],\n",
      "        [0.5756],\n",
      "        [0.4951],\n",
      "        [0.4000],\n",
      "        [0.4852],\n",
      "        [0.4504],\n",
      "        [0.4720],\n",
      "        [0.5601],\n",
      "        [0.5362],\n",
      "        [0.5297],\n",
      "        [0.5649],\n",
      "        [0.4720],\n",
      "        [0.5066],\n",
      "        [0.6362],\n",
      "        [0.5816]], grad_fn=<SigmoidBackward>)\n"
     ]
    }
   ],
   "source": [
    "# 测试一下模型\n",
    "for fea, label in iter(dl_train):\n",
    "    out = net(fea)\n",
    "    print(out)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型的训练\n",
    "模型的训练一般会有三种风格， 由于计算AUC的时候， 只有一类会报错， 这里训练的时候得处理一下， 所以采用脚本的训练风格以方便处理， 但是在训练之前， 先把训练的相关设置定义好， 包括评估指标， 损失函数和优化方法等"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:15:28.213036Z",
     "start_time": "2020-10-07T02:15:28.199074Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的相关设置\n",
    "def auc(y_pred, y_true):\n",
    "    pred = y_pred.data\n",
    "    y = y_true.data\n",
    "    return roc_auc_score(y, pred)     # 计算AUC， 但要注意如果y只有一个类别的时候， 会报错\n",
    "\n",
    "loss_func = nn.BCELoss()\n",
    "optimizer = torch.optim.Adam(params=net.parameters(), lr=0.001)\n",
    "metric_func = auc\n",
    "metric_name = 'auc'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 脚本训练方式 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:19:12.204603Z",
     "start_time": "2020-10-07T02:19:05.947125Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start Training...\n",
      "========================================================================2020-10-07 10:19:05\n",
      "[step = 10] loss: 0.075, auc: 0.993\n",
      "[step = 20] loss: 0.059, auc: 0.993\n",
      "[step = 30] loss: 0.056, auc: 0.995\n",
      "[step = 40] loss: 0.064, auc: 0.995\n",
      "[step = 50] loss: 0.085, auc: 0.993\n",
      "[step = 60] loss: 0.082, auc: 0.994\n",
      "[step = 70] loss: 0.084, auc: 0.990\n",
      "[step = 80] loss: 0.083, auc: 0.991\n",
      "\n",
      "EPOCH = 1, loss = 0.083,auc  = 0.991, val_loss = 0.961, val_auc = 0.594\n",
      "\n",
      "================================================================================2020-10-07 10:19:07\n",
      "[step = 10] loss: 0.057, auc: 1.000\n",
      "[step = 20] loss: 0.048, auc: 0.999\n",
      "[step = 30] loss: 0.048, auc: 0.999\n",
      "[step = 40] loss: 0.052, auc: 1.000\n",
      "[step = 50] loss: 0.058, auc: 0.999\n",
      "[step = 60] loss: 0.065, auc: 0.997\n",
      "[step = 70] loss: 0.063, auc: 0.998\n",
      "[step = 80] loss: 0.061, auc: 0.998\n",
      "\n",
      "EPOCH = 2, loss = 0.061,auc  = 0.998, val_loss = 1.344, val_auc = 0.643\n",
      "\n",
      "================================================================================2020-10-07 10:19:09\n",
      "[step = 10] loss: 0.052, auc: 0.998\n",
      "[step = 20] loss: 0.074, auc: 0.993\n",
      "[step = 30] loss: 0.073, auc: 0.995\n",
      "[step = 40] loss: 0.065, auc: 0.996\n",
      "[step = 50] loss: 0.065, auc: 0.996\n",
      "[step = 60] loss: 0.058, auc: 0.997\n",
      "[step = 70] loss: 0.052, auc: 0.997\n",
      "[step = 80] loss: 0.049, auc: 0.997\n",
      "\n",
      "EPOCH = 3, loss = 0.049,auc  = 0.997, val_loss = 1.358, val_auc = 0.642\n",
      "\n",
      "================================================================================2020-10-07 10:19:10\n",
      "[step = 10] loss: 0.039, auc: 1.000\n",
      "[step = 20] loss: 0.030, auc: 1.000\n",
      "[step = 30] loss: 0.050, auc: 0.998\n",
      "[step = 40] loss: 0.041, auc: 0.999\n",
      "[step = 50] loss: 0.058, auc: 0.998\n",
      "[step = 60] loss: 0.063, auc: 0.998\n",
      "[step = 70] loss: 0.068, auc: 0.998\n",
      "[step = 80] loss: 0.069, auc: 0.998\n",
      "\n",
      "EPOCH = 4, loss = 0.069,auc  = 0.998, val_loss = 0.871, val_auc = 0.671\n",
      "\n",
      "================================================================================2020-10-07 10:19:12\n",
      "Finished Training...\n"
     ]
    }
   ],
   "source": [
    "epochs = 4\n",
    "log_step_freq = 10\n",
    "\n",
    "dfhistory = pd.DataFrame(columns=[\"epoch\", \"loss\", metric_name, \"val_loss\", \"val_\"+metric_name])\n",
    "print('Start Training...')\n",
    "nowtime = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "print('========='*8 + \"%s\" %nowtime)\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    # 训练阶段\n",
    "    net.train()\n",
    "    loss_sum = 0.0\n",
    "    metric_sum = 0.0\n",
    "    step = 1\n",
    "    \n",
    "    for step, (features, labels) in enumerate(dl_train, 1):\n",
    "        \n",
    "        # 梯度清零\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # 正向传播\n",
    "        predictions = net(features)\n",
    "        loss = loss_func(predictions, labels)\n",
    "        try:          # 这里就是如果当前批次里面的y只有一个类别， 跳过去\n",
    "            metric = metric_func(predictions, labels)\n",
    "        except ValueError:\n",
    "            pass\n",
    "        \n",
    "        # 反向传播求梯度\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # 打印batch级别日志\n",
    "        loss_sum += loss.item()\n",
    "        metric_sum += metric.item()\n",
    "        if step % log_step_freq == 0:\n",
    "            print((\"[step = %d] loss: %.3f, \"+metric_name+\": %.3f\") %\n",
    "                  (step, loss_sum/step, metric_sum/step))\n",
    "    \n",
    "    # 验证阶段\n",
    "    net.eval()\n",
    "    val_loss_sum = 0.0\n",
    "    val_metric_sum = 0.0\n",
    "    val_step = 1\n",
    "    \n",
    "    for val_step, (features, labels) in enumerate(dl_vaild, 1):\n",
    "        with torch.no_grad():\n",
    "            predictions = net(features)\n",
    "            val_loss = loss_func(predictions, labels)\n",
    "            try:\n",
    "                val_metric = metric_func(predictions, labels)\n",
    "            except ValueError:\n",
    "                pass\n",
    "        val_loss_sum += val_loss.item()\n",
    "        val_metric_sum += val_metric.item()\n",
    "    \n",
    "    # 记录日志\n",
    "    info = (epoch, loss_sum/step, metric_sum/step, val_loss_sum/val_step, val_metric_sum/val_step)\n",
    "    dfhistory.loc[epoch-1] = info\n",
    "    \n",
    "    # 打印epoch级别日志\n",
    "    print((\"\\nEPOCH = %d, loss = %.3f,\"+ metric_name + \\\n",
    "          \"  = %.3f, val_loss = %.3f, \"+\"val_\"+ metric_name+\" = %.3f\") \n",
    "          %info)\n",
    "    nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n",
    "    print(\"\\n\"+\"==========\"*8 + \"%s\"%nowtime)\n",
    "        \n",
    "print('Finished Training...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:20:47.390136Z",
     "start_time": "2020-10-07T02:20:47.372138Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>loss</th>\n",
       "      <th>auc</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>val_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.083403</td>\n",
       "      <td>0.991398</td>\n",
       "      <td>0.960851</td>\n",
       "      <td>0.593605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.060843</td>\n",
       "      <td>0.997879</td>\n",
       "      <td>1.343973</td>\n",
       "      <td>0.643192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.049074</td>\n",
       "      <td>0.997406</td>\n",
       "      <td>1.357822</td>\n",
       "      <td>0.641545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.069463</td>\n",
       "      <td>0.998283</td>\n",
       "      <td>0.871493</td>\n",
       "      <td>0.671060</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch      loss       auc  val_loss   val_auc\n",
       "0    1.0  0.083403  0.991398  0.960851  0.593605\n",
       "1    2.0  0.060843  0.997879  1.343973  0.643192\n",
       "2    3.0  0.049074  0.997406  1.357822  0.641545\n",
       "3    4.0  0.069463  0.998283  0.871493  0.671060"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfhistory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:20:57.841101Z",
     "start_time": "2020-10-07T02:20:57.554229Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXgUVfr//fcNYRVwAUQEIaggLghoRBxccEdF8HFBFFQckVFHxRk3lBlH+YrrPOM2KOKuxIWBUVEZ0UERF0CCIsqOiBDXgGwOBAjcvz9Ox3SSTtKBdDpJf17X1Ve661RX39WVrrvOOVWnzN0REZHUVSvZAYiISHIpEYiIpDglAhGRFKdEICKS4pQIRERSnBKBiEiKUyKQCmVmtc3sVzNrU5HzJpOZ7W9mFX6etZmdZGbLo14vMrNj4pl3Bz7rSTO7dUffX8py7zSzZyt6uVK50pIdgCSXmf0a9bIhsBnYFnn9B3fPLM/y3H0b0Kii500F7n5ARSzHzAYDA929Z9SyB1fEsqVmUiJIce7+2444csQ52N3/W9L8Zpbm7nmVEZuIVA41DUmpIlX/V8zsJTPbAAw0s6PMbIaZrTWzH8zsYTOrE5k/zczczNIjr8dGyv9jZhvMbLqZtSvvvJHy08xssZmtM7NHzOxjMxtUQtzxxPgHM1tqZmvM7OGo99Y2swfMbLWZfQ30KuX7+YuZvVxk2igz+0fk+WAzWxBZn68jR+slLSvbzHpGnjc0sxcisc0DDo/xucsiy51nZn0i0zsB/wSOiTS7rYr6bm+Pev8VkXVfbWavmVnLeL6bspjZWZF41prZe2Z2QFTZrWb2vZmtN7OFUeva3cw+i0z/yczuj/fzpIK4ux564O4Ay4GTiky7E9gCnEk4cGgAHAEcSahR7gssBq6OzJ8GOJAeeT0WWAVkAHWAV4CxOzDvnsAGoG+k7M/AVmBQCesST4yvA7sC6cAv+esOXA3MA1oDTYFp4acS83P2BX4Fdola9s9ARuT1mZF5DDgB2AQcGik7CVgetaxsoGfk+d+BqcDuQFtgfpF5+wEtI9vkwkgMLSJlg4GpReIcC9weeX5KJMYuQH3gUeC9eL6bGOt/J/Bs5PmBkThOiGyjWyPfex3gYOBbYK/IvO2AfSPPZwEXRJ43Bo5M9m8h1R6qEUg8PnL3N9x9u7tvcvdZ7j7T3fPcfRkwBjiulPePd/csd98KZBJ2QOWdtzcwx91fj5Q9QEgaMcUZ493uvs7dlxN2uvmf1Q94wN2z3X01cE8pn7MM+IqQoABOBta6e1ak/A13X+bBe8AUIGaHcBH9gDvdfY27f0s4yo/+3HHu/kNkm7xISOIZcSwXYADwpLvPcfdcYBhwnJm1jpqnpO+mNP2Bie7+XmQb3QM0ISTkPELSOTjSvPhN5LuDkNDbm1lTd9/g7jPjXA+pIEoEEo+V0S/MrKOZvWVmP5rZemAE0KyU9/8Y9XwjpXcQlzTv3tFxuLsTjqBjijPGuD6LcCRbmheBCyLPLyQksPw4epvZTDP7xczWEo7GS/uu8rUsLQYzG2RmX0SaYNYCHeNcLoT1+2157r4eWAO0ipqnPNuspOVuJ2yjVu6+CLiesB1+jjQ17hWZ9VLgIGCRmX1qZqfHuR5SQZQIJB5FT518nHAUvL+7NwFuIzR9JNIPhKYaAMzMKLzjKmpnYvwB2CfqdVmnt74CnBQ5ou5LSAyYWQNgPHA3odlmN+CdOOP4saQYzGxf4DHgSqBpZLkLo5Zb1qmu3xOam/KX15jQBPVdHHGVZ7m1CNvsOwB3H+vuPQjNQrUJ3wvuvsjd+xOa//5/YIKZ1d/JWKQclAhkRzQG1gH/M7MDgT9Uwme+CRxmZmeaWRowFGieoBjHAdeZWSszawrcXNrM7v4T8BHwDLDI3ZdEiuoBdYEcYJuZ9QZOLEcMt5rZbhaus7g6qqwRYWefQ8iJgwk1gnw/Aa3zO8djeAm4zMwONbN6hB3yh+5eYg2rHDH3MbOekc++kdCvM9PMDjSz4yOftyny2EZYgYvMrFmkBrEusm7bdzIWKQclAtkR1wOXEH7kjxOOiBMqsrM9H/gHsBrYD/iccN1DRcf4GKEt/0tCR+b4ON7zIqHz98WomNcCfwJeJXS4nktIaPH4G6Fmshz4D/B81HLnAg8Dn0bm6QhEt6u/CywBfjKz6Cae/Pe/TWiieTXy/jaEfoOd4u7zCN/5Y4Qk1QvoE+kvqAfcR+jX+ZFQA/lL5K2nAwssnJX2d+B8d9+ys/FI/Cw0tYpUL2ZWm9AUca67f5jseESqM9UIpNows15mtmukeeGvhDNRPk1yWCLVnhKBVCdHA8sIzQu9gLPcvaSmIRGJk5qGRERSnGoEIiIprtoNOtesWTNPT09PdhgiItXK7NmzV7l7zFOuE5YIzOxpwrAAP7v7IaXMdwQwg3DKWJmn6aWnp5OVlVVxgYqIpAAzK/EK+UQ2DT1LKaM2wm+nAN4LTE5gHCIiUoqEJQJ3n0a4iKY01wATCCMhiohIEiSts9jMWgH/HzA6jnmHmFmWmWXl5OQkPjgRkRSSzLOGHgRu9nC7wlK5+xh3z3D3jObNSxteRkREyiuZZw1lAC+HQSRpBpxuZnnu/loSYxIRSTlJqxG4ezt3T3f3dMKgXlcpCYgkWWYmpKdDrVrhb2ZmWe+QGiCRp4++BPQEmplZNmE0xToA7l5mv4CIVLLMTBgyBDZuDK+//Ta8Bhiw04OTShVW7YaYyMjIcF1HUENkZsLw4bBiBbRpAyNHaocTD3fIy4OtW2HLlvA3/xH9urxlt98Oa9cW/7y2bWH58speS6lgZjbb3WPezrTaXVksNUQyjz63b9/5nWYyy/LyEvv9FLViReV+nlQ6JQJJjuHDC5JAvo0b4Y9/hIULK2anWdK82yvh5ldpaVCnDtStG/7mP6JfRz9v2DD+ectTVp55u3aFlSuLr8vuu4daiCX6bqSSLGoakuSoVSvsXGIxq/ydYEUtJ/9RHXeaRWtpELbT9u3Qrx88/jjstlvy4pOdoqYhqTp+/BH+8peSk0CbNqGZSCpffpNcdL/N//0ffPdd2GYzZ8KLL8LvfpfcOKXCaRhqqRy5uXDPPdC+PTz3HJx2GjRoUHiehg3hrruSE58EAwaEjuHt28Pfiy6CYcPg449D7eDYY0Ny2FbmdaBSjSgRSGK5w/jxcOCBcMstcMIJMG8eTJoETzwRzkgxC3/HjNFZQ1XVkUfCnDlw/vlw221hO8bqT5BqSYlAEuezz+C44+C886BRI3j3XXj9dejQIZQXPfpUEqjamjSBsWNDje6zz6BzZ3j11WRHJRVAiUAq3g8/wO9/DxkZsGABjB4Nn38OJ52U7MhkZ5nBxReH7bnffnD22XDFFcXPAJNqRYlAKk5uLtx9dzjiHzsWrr8eli6FP/whnE4pNcf++4d+g5tuCmcTHXEEzJ2b7KhkBykRyM5zh3/9Czp2hFtvDUf+8+fD/ffDrrsmOzpJlLp14d574Z134JdfoFs3+Oc/Sz4jTKosJQLZOVlZ4UySfv3CTn/KlNBuvP/+yY5MKsvJJ8MXX8CJJ8I110DfvrBqVbKjknJQIpAd8/33MGhQaBJYtCic8fPZZ+FsEkk9e+4Jb74JDz4IkyeHjuT33kt2VBInJQIpn02b4M47Qz/ASy+FNuIlS+Dyy6F27WRHJ8lkBkOHhgvPGjcOTYS33BKG9ZAqTYlA4uMOr7wS+gH++lc49dTQD3DvveoHkMK6dIHZs+Gyy8JFhEcfDcuWJTsqKYUSgZRt1qzwY+7fH/bYA95/HyZMCKcPisSyyy7hgsFx40LTYZcuuslNFaZEICX77ju45JJwNsjSpfDkk6FzuGfPZEcm1cV554WO5EMPhYEDw//Thg3JjkqKUCKQ4jZuDOPJdOgAL78cxppZsiRU9dUPIOXVti1MnRqGphg7Fg47LBxQSJWhRCAF3EMHcMeO4Ud7+unhyuC77w7DC4jsqLQ0uOOOkBA2b4ajjgrXmVTGvSGkTEoEEsycGYYXvvBCaNYMPvggXCS2777JjkxqkmOOCU1FffuGM8569QpDkkhSKRGkuuzsMNRw9+5h4Lennw6dw8cem+zIpKbaffdwkPH44/DRR+Gag0mTkh1VSlMiSFUbN4aqeocO4Ud5662weDFceqn6ASTxzMLd0LKyYK+94Iwz4LrrQrORVLqEJQIze9rMfjazr0ooH2BmcyOPT8ysc6JikSjbt4fT+A44AG6/Hc48M9wjeOTIcBGQSGU66CD49NMwNMVDD4Wa6cKFyY4q5SSyRvAs0KuU8m+A49z9UOD/gDEJjEUAZswI/QADB0KLFjBtWrhILD092ZFJKqtfHx5+GN54IzRVHn44PPWUBq+rRAlLBO4+DfillPJP3H1N5OUMoHWiYkl5K1eGm74cdVS4F+2zz4ajsGOOSXZkIgV69w4dyUcdBYMHh7uhrV2b7KhSQlXpI7gM+E9JhWY2xMyyzCwrJyenEsOq5v73P/jb30Iz0L//HW5AvnhxuKinVlXZ9CJR9t47DGt9zz1hFNvOncN9DyShkr43MLPjCYng5pLmcfcx7p7h7hnNmzevvOCqq+3b4YUXQkfwiBHhVL2FC8NFYo0aJTs6kdLVqgU33xwSQFpaOINtxAjYti3ZkdVYSU0EZnYo8CTQ191XJzOWGuOTT0KH28UXQ6tW4fS8l14KV3eKVCfduoVbYl54YajZHn98aNqUCpe0RGBmbYB/Axe5++JkxVFjfPstXHAB9OgRxgh6/vnQOdyjR7IjE9lxTZqE2u3zz4ek0LlzGPBQKlQiTx99CZgOHGBm2WZ2mZldYWZXRGa5DWgKPGpmc8xMg4/siF9/DcNCd+wIr70Wni9aFC4SUz+A1BQXXRQSQfv2cO654T7YGzcmO6oaw7yanaKVkZHhWRqwqqAf4JZbwiX6F14YxgRq0ybZkYkkzpYt4WDnvvvgwANDs2dnXYIUDzOb7e4Zscp0yFgdffRRaD8dNAj22Sf0C2RmKglIzVe3brgZ0rvvwpo1cOSR8MgjuuZgJykRVCfLl4dzq485Bn78MQzpO316OO9aJJWcdBLMnRv+Xnst9OkDOrV8hykRVAcbNsDw4aEf4I03wtAQixaFi8TUDyCpqnnz8Ht46KFw7UHnzjBlSrKjqpa0F6nKtm+HZ54J1wPcdVfoJFu8OJxKt8suyY5OJPnMQo1g5sxw7+yTTw79Zlu3JjuyakWJoKqaNg2OOAJ+//twDcD06aEpqLVG4hAppkuXMJLp4MHhquSjj4avv052VNWGEkFV88034T6vxx0HP/8cOoGnTw8XiYlIyXbZBcaMCcOqL14MXbuG34+USYmgqli/PlRpO3YMN+m4447QD3DhhaH6KyLxOffcMHhd585hpN2LLw79bFIiJYJk27YtDLnboUOo0vbvH45mbrsNGjZMdnQi1VObNvD+++HEiszMUDuYNSvZUVVZSgTJ9MEHkJER2jX32y90eD33XBgjSER2TlpaOLFi6tRwIdrvfhcuRNu+PdmRVTlKBMmwbBmccw707AmrV4erI/MvEhORinXMMaGp6Kyzwqimp54arsaX3ygRVKb168M/4oEHwttvh2GhFy0KzUHqBxBJnN13h3Hj4IknwvDWhx4Kb72V7KiqDCWCyrBtW/gHbN8+VE0vuACWLAk3imnQINnRiaQGs9AMO3t2aH7t3RuGDoXc3GRHlnRKBIn2/vtw2GEwZEhIBLNmhVtF7r13siMTSU0HHhiGaL/22nCv5O7dYcGCZEeVVEoEifL113D22XDCCbBuXbhJ/Icfhs5hEUmu+vXD0BRvvBHu33H44aHWnqKD1ykRVLR16+Cmm8JRxzvvwMiR4WijXz/1A4hUNb17h8HrevQItfZ+/cKopilGiaCibNsGjz8emn/+/vdwIcuSJXDrreoHEKnKWraEyZPD8NavvRaGq/joo2RHVamUCCrClCnhgpUrrghXBs+aBU8/Hf7BRKTqq1Ur1OQ//hjq1AlDvNxxB+TlJTuySqFEsDOWLIG+fcOY6Bs2hDFOPvggtDeKSPXTrVu4JeaAAeGq5OOPhxUrkh1VwikR7Ii1a+H66+Hgg+G998ItIhcsCGOcqB9ApHpr3Biefz7cCnbOnDBm0fjxyY4qoZQIyiMvD0aPDv0ADzwQBrNasgSGDQtnIYhIzTFwYEgE7duHEYGHDIGNG5MdVUIoEcTr3XdDP8CVV4aawOzZ8OSTsNdeyY5MRBJlv/1Cx/HNN4ffe0ZGGK6ihklYIjCzp83sZzP7qoRyM7OHzWypmc01s8MSFctOWbw43A/1lFPgf/+DCRPCRWJduyY7MhGpDHXrhpGB3303NAt36waPPFKjrjlIZI3gWaBXKeWnAe0jjyHAYwmMpfzWrIE//zkc/U+dGk4tmz8/XCSmfgCR1HPiiaE2cPLJ4arkPn0gJyfZUVWIhCUCd58G/FLKLH2B5z2YAexmZsk/3zIvD0aNCu2CDz4Il14a+gFuukn9ACKprnnzcDXyww+HGkLnzvDf/yY7qp2WzD6CVsDKqNfZkWnFmNkQM8sys6ycRGbgd94JG/bqq6FTJ/jss3DruxYtEveZIlK9mME114T7h+y6a2g2HjYMtm5NdmQ7LJmJIFb7SsxGN3cf4+4Z7p7RvHnzio9k4cJwqfmpp8LmzfDqq+G00C5dKv6zRKRm6Nw5nDRy+eWh6bhHjzDGWDWUzESQDewT9bo18H2lRvDLL3DddeHo/8MP4f77Yd68cAML9QOISFkaNgxDy4wfH5qQu3SBsWOTHVW5JTMRTAQujpw91B1Y5+6JuW1QZiakp4fLyNPTw8Ui//xn6Ad45BG47LKwEW+4AerVS0gIIlKDnXNO6Eju2hUuuig81q9PdlRxM0/QKVBm9hLQE2gG/AT8DagD4O6jzcyAfxLOLNoIXOruWWUtNyMjw7OyypytQGZm8QtBzMKpXyecEC4MO/TQ+JcnIlKSvDy4664wTlG7duE2tEcckeyoADCz2e4ecxz8hCWCRCl3IkhPh2+/LT69eXP46Sc1AYlIxfvoozBe0fffw513wo03hhaJJCotEdT8K4tLGjBq1SolARFJjKOPDsNTnHVWOKPolFNCUqiian4iaNOmfNNFRCrC7rvDuHHhzmfTp4ezjN58M9lRxVTzE8HIkaFnP1rDhmG6iEgimcHgweE001at4Mwzw1XJubnJjqyQmp8IBgwIF4W1bRs2Stu24fWAAcmOTERSRceOMGMGDB0azlQ88sgwdH0VUfM7i0VEqpK33oJBg8Iglg89FGoMldBfmdqdxSIiVckZZ8DcueFK5CFDwr0O1qxJakhKBCIila1lS5g8Ge67D15/PXQkf/hh0sJRIhARSYZatcL1BZ98Eu550LNnuE9yXl7lh1LpnygiIgWOOAI+/zzcGvOOO0JCiHURbAIpEYiIJFvjxvDcc2HAurlzw+B148dX2scrEYiIVBUDBoTaQYcOoRN5yJBwdlGCKRGIiFQl++0Xxiq65RZ48knIyIC77y48gnJmZoV+pK4jEBGpqt57L9wnfd26wtMbNiz3hbG6jkBEpDo64YTQf1DUxo0wfHiFfYwSgYhIVfbdd7GnlzSy8g5QIhARqcoqYQRlJQIRkaqsEkZQViIQEanKKmEE5bQKW5KIiCTGgAEJHTpfNQIRkRSnRCAikuISmgjMrJeZLTKzpWY2LEZ5GzN738w+N7O5ZnZ6IuMREZHiEtZHYGa1gVHAyUA2MMvMJrr7/KjZ/gKMc/fHzOwgYBKQnqiYRKRq2rp1K9nZ2eRWsXv5Vkf169endevW1KlTJ+73JLKzuBuw1N2XAZjZy0BfIDoRONAk8nxX4PsExiMiVVR2djaNGzcmPT0dq4TbNtZU7s7q1avJzs6mXbt2cb8vkU1DrYCVUa+zI9Oi3Q4MNLNsQm3gmlgLMrMhZpZlZlk5OTmJiFVEkig3N5emTZsqCewkM6Np06blrlklMhHE2qJFR7i7AHjW3VsDpwMvmFmxmNx9jLtnuHtG8+bNExCqiCSbkkDF2JHvMZGJIBvYJ+p1a4o3/VwGjANw9+lAfaBZAmMSEZEiEpkIZgHtzaydmdUF+gMTi8yzAjgRwMwOJCQCtf2ISKkyMyt2eP61a9fy6KOPlvt9p59+OmvXri33+wYNGsT4SrwDWVkSlgjcPQ+4GpgMLCCcHTTPzEaYWZ/IbNcDl5vZF8BLwCCvbjdIEJFKlZkZbtz17bfgHv4OGbJzyaCkRLBt27ZS3zdp0iR22223Hf/gKiKhQ0y4+yRCJ3D0tNuins8HeiQyBhGpfnr2LD6tXz+46qpw466NGwuXbdwIQ4eGURhWrYJzzy1cPnVq6Z83bNgwvv76a7p06UKdOnVo1KgRLVu2ZM6cOcyfP5+zzjqLlStXkpuby9ChQxkyZAgA6enpZGVl8euvv3Laaadx9NFH88knn9CqVStef/11GjRoUOa6TpkyhRtuuIG8vDyOOOIIHnvsMerVq8ewYcOYOHEiaWlpnHLKKfz973/nX//6F3fccQe1a9dm1113Zdq0aWUuPx4aa0hEqpXs7NjTV6/e8WXec889fPXVV8yZM4epU6dyxhln8NVXX/12CubTTz/NHnvswaZNmzjiiCM455xzaNq0aaFlLFmyhJdeeoknnniCfv36MWHCBAYOHFjq5+bm5jJo0CCmTJlChw4duPjii3nssce4+OKLefXVV1m4cCFm9lvz04gRI5g8eTKtWrXaoSapkigRiEiVU9oRfJs2oTmoqLZtw99mzcquAZSlW7duhc7Df/jhh3n11VcBWLlyJUuWLCmWCNq1a0eXLl0AOPzww1m+fHmZn7No0SLatWtHhw4dALjkkksYNWoUV199NfXr12fw4MGcccYZ9O7dG4AePXowaNAg+vXrx9lnn71zKxlFYw2JSLVSCcPzs8suu/z2fOrUqfz3v/9l+vTpfPHFF3Tt2jXmefr16tX77Xnt2rXJy8sr83NK6hJNS0vj008/5ZxzzuG1116jV69eAIwePZo777yTlStX0qVLF1bvTDUoSlyJwMyGmlkTC54ys8/M7JQKiUBEpBwSMTx/48aN2bBhQ8yydevWsfvuu9OwYUMWLlzIjBkzdvyDiujYsSPLly9n6dKlALzwwgscd9xx/Prrr6xbt47TTz+dBx98kDlz5gDw9ddfc+SRRzJixAiaNWvGypUrS1t83OJtGvq9uz9kZqcCzYFLgWeAdyokChGRcqjo4fmbNm1Kjx49OOSQQ2jQoAEtWrT4raxXr16MHj2aQw89lAMOOIDu3btX2OfWr1+fZ555hvPOO++3zuIrrriCX375hb59+5Kbm4u788ADDwBw4403smTJEtydE088kc6dO1dIHBbP2ZpmNtfdDzWzh4Cp7v6qmX3u7l0rJIpyyMjI8KysrMr+WBFJoAULFnDggQcmO4waI9b3aWaz3T0j1vzx9hHMNrN3CMNATDazxsD2nYpURESqhHibhi4DugDL3H2jme1BaB4SEZES/PGPf+Tjjz8uNG3o0KFcemnV2n3GmwiOAua4+//MbCBwGPBQ4sISEan+Ro0alewQ4hJv09BjwEYz6wzcBHwLPJ+wqEREpNLEmwjyImMA9QUecveHgMaJC0tERCpLvE1DG8zsFuAi4JjIbSjjvw+aiIhUWfHWCM4HNhOuJ/iRcKex+xMWlYiIVJq4EkFk558J7GpmvYFcd1cfgYgkR0XfkGAHNGrUqMSy5cuXc8ghh1RiNDsn3iEm+gGfAucB/YCZZnZu6e8SEUmARNyQIMXF20cwHDjC3X8GMLPmwH+BqnOLHRGpGa67DiJj68Q0YwZs3lx42saNcNll8MQTsd/TpQs8+GCpH3vzzTfTtm1brrrqKgBuv/12zIxp06axZs0atm7dyp133knfvn3Lszbk5uZy5ZVXkpWVRVpaGv/4xz84/vjjmTdvHpdeeilbtmxh+/btTJgwgb333pt+/fqRnZ3Ntm3b+Otf/8r5559frs/bEfEmglr5SSBiNRq5VESSoWgSKGt6nPr378911133WyIYN24cb7/9Nn/6059o0qQJq1atonv37vTp06dcN4jPv5bgyy+/ZOHChZxyyiksXryY0aNHM3ToUAYMGMCWLVvYtm0bkyZNYu+99+att94CwoB3lSHeRPC2mU0m3E4SQufxpFLmFxHZMWUcuZOeXvINCXbiRgRdu3bl559/5vvvvycnJ4fdd9+dli1b8qc//Ylp06ZRq1YtvvvuO3766Sf22muvuJf70Ucfcc011wBhtNG2bduyePFijjrqKEaOHEl2djZnn3027du3p1OnTtxwww3cfPPN9O7dm2OOOWaH16c84u0svhEYAxwKdAbGuPvNiQxMRCSmBN6Q4Nxzz2X8+PG88sor9O/fn8zMTHJycpg9ezZz5syhRYsWMe9FUJqSBva88MILmThxIg0aNODUU0/lvffeo0OHDsyePZtOnTpxyy23MGLEiJ1ep3jEfYcyd58ATEhgLCIiZcsff3r4cFixItyybOTIChmXun///lx++eWsWrWKDz74gHHjxrHnnntSp04d3n//fb6NVRMpw7HHHktmZiYnnHACixcvZsWKFRxwwAEsW7aMfffdl2uvvZZly5Yxd+5cOnbsyB577MHAgQNp1KgRzz777E6vUzxKTQRmtgGIlc4McHdvkpCoRERKU9E3JIg4+OCD2bBhA61ataJly5YMGDCAM888k4yMDLp06ULHjh3LvcyrrrqKK664gk6dOpGWlsazzz5LvXr1eOWVVxg7dix16tRhr7324rbbbmPWrFnceOON1KpVizp16vDYY49V+DrGEtf9CKoS3Y9ApObR/QgqVqLuR7BDzKyXmS0ys6VmNqyEefqZ2Xwzm2dmLyYyHhERKS7uPoLyioxHNAo4GcgGZpnZRHefHzVPe+AWoIe7rzGzPRMVj4hIRfvyyy+56KKLCk2rV68eM2fOTFJEOyZhiQDoBix192UAZvYyYfTS+VHzXA6Mcvc1AEWuVRCRFOLu5To/vyro1KnTbzeWryp2pLk/kU1DrYCVUa+zI9OidQA6mNnHZjbDzHrFWpeK/ysAABFaSURBVJCZDTGzLDPLysnJSVC4IpIs9evXZ/Xq1Tu0E5MC7s7q1aupX79+ud6XyBpBrNRedCunAe2BnkBr4EMzO8Td1xZ6k/sYwnUMZGRk6D9FpIZp3bo12dnZ6EBv59WvX5/WrVuX6z2JTATZwD5Rr1sD38eYZ4a7bwW+MbNFhMQwK4FxiUgVU6dOHdq1a5fsMFJWIpuGZgHtzaydmdUF+gMTi8zzGnA8gJk1IzQVLUtgTCIiUkTCEoG75wFXA5OBBcA4d59nZiPMrE9ktsnAajObD7wP3OjuqxMVk4iIFKcLykREUkDSLigTEZGqT4lARCTFKRGIiKQ4JQIRkRSnRCAikuKUCEREUpwSgYhIilMiEBFJcUoEIiIpTolARCTFKRGIiKQ4JQIRkRSnRCAikuKUCEREUpwSgYhIilMiEBFJcUoEIiIpTolARCTFKRGIiKQ4JQIRkRSnRCAikuISmgjMrJeZLTKzpWY2rJT5zjUzN7OMRMYjIiLFJSwRmFltYBRwGnAQcIGZHRRjvsbAtcDMRMUiIiIlS2SNoBuw1N2XufsW4GWgb4z5/g+4D8hNYCwiIlKCRCaCVsDKqNfZkWm/MbOuwD7u/mZpCzKzIWaWZWZZOTk5FR+piEgKS2QisBjT/LdCs1rAA8D1ZS3I3ce4e4a7ZzRv3rwCQxQRkUQmgmxgn6jXrYHvo143Bg4BpprZcqA7MFEdxiIilSuRiWAW0N7M2plZXaA/MDG/0N3XuXszd09393RgBtDH3bMSGJOIiBSRsETg7nnA1cBkYAEwzt3nmdkIM+uTqM8VEZHySUvkwt19EjCpyLTbSpi3ZyJjERGR2HRlsYhIilMiEBFJcUoEIiIpTolARCTFKRGIiKQ4JQIRkRSnRCAikuKUCEREUpwSgYhIilMiEBFJcUoEIiIpTolARCTFKRGIiKQ4JQIRkRSnRCAikuKUCEREUpwSgYhIilMiEBFJcUoEIiIpTolARCTFKRGIiKS4hCYCM+tlZovMbKmZDYtR/mczm29mc81sipm1TWQ8IiJSXMISgZnVBkYBpwEHAReY2UFFZvscyHD3Q4HxwH2JikdERGJLZI2gG7DU3Ze5+xbgZaBv9Azu/r67b4y8nAG0TmA8IiISQyITQStgZdTr7Mi0klwG/CdWgZkNMbMsM8vKycmpwBBFRCSRicBiTPOYM5oNBDKA+2OVu/sYd89w94zmzZtXYIgiIpKWwGVnA/tEvW4NfF90JjM7CRgOHOfumxMYj4iIxJDIGsEsoL2ZtTOzukB/YGL0DGbWFXgc6OPuPycwFhERKUHCEoG75wFXA5OBBcA4d59nZiPMrE9ktvuBRsC/zGyOmU0sYXEiIpIgiWwawt0nAZOKTLst6vlJifx8EREpm64sFhFJcUoEIiIpTolARCTFKRGIiKQ4JQIRkRSnRCAikuKUCEREUpwSgYhIilMiEBFJcSmRCDIzIT0datUKfzMzkx2RiEjVkdAhJqqCzEwYMgQ2Rm5/8+234TXAgAHJi0tEpKow95i3CKiyMjIyPCsrK+7509PDzr+oWrVg772hSROYNy9Mu/demDoV6teHBg3C36ZN4f7IXRL+/W/45ptQll++xx5w6qmhfNEi2Ly54P0NGkDDhuEhIrKjMjNh+HBYsQLatIGRI8t/IGtms909I1ZZja8RrFgRe/r27XDyyRCdBzdvhl9+gU2bIDc3/G3SpCARPPMMvPlm4eXsvz8sWRKeX3FFSCTRunSBzz8Pz084ISSd/CRSvz507w6jR4fyoUNh9erCiaRTJ7joolD+wguwbVvB+xs0gH32gQMPDOXffAN16xZefq0q3PhXEf/cIjVdZbRqpGyNoG1bWL68fJ+9ZUtBgsjNDQ936NgxlH/yCfzwQ8E8mzbB7rvDwIGh/L77ws46vyw3N+zE77knlPfqBUuXFk5Ep50GEyaE8ubNYdWqwjENGABjx4bnDRuG90S76ioYNSokvv33L1zbadAALrwQfv/78E923XWFyxo0gOOOC8lq48aQBKNrQw0ahO+xWTPIy4Nffw3T69UDi3V/uihF/7nz4x8zRslAqjb3sC+oUyccaK1bBz/9VLBP2Lw5/D322PBb+PxzmD27oDz/cdtt4cDt5Zdh0qTCZVu2FBxU7rorrF9fPI7y7sNSukYwcmTsHc7IkeVfVt264dGkSezy3/2u9PffdFPp5W+/XXr5l18WJJH8ZLHHHgXlY8aE9YxORIcfHsry8uDoowsnmfx/Wgiv33ij+PS77w6J4Mcf4fzzi8f0yCNw9dUwfz507lwwPT9RPPoo9O8Pc+bA4MEFieSjj4onrY0b4corw3qmpYUf2kUXwb77wuLF4fvJn57/t1evkIhWrICvvipclpYWYmrQINT0Vq0qXr7HHuHHvG1bSF5VuQZVGap6Lc0dtm4N/6P16oXHxo3w9deFd6SbN4f//ZYtw4HgG28U3knn5ob9Qvv2MH06PPhg8R31M8/AQQfBiy/Cn/9c+L0ACxaEg8CnnoLrry8e68qV0Lp1+Oy//a1wmRnccEPYn3z9dfg95Nfi8w+mtm2D2rVjJwEoubVjR9T4RJD/T1yV/7njtddepZfn1zxiqVsXnn++5PKmTUNtJt/27eEfPn/H2Lp12NFGJ6FNm+CQQwpi+8c/CieZTZvCThzCP3SLFmHahg3Fk0C+DRvCj3Lr1hDDsceGZcyeHZrOivr005AI3n4b/vCH4uXz54da17PPlv5jHTky/FjNCieLFStC4r/rLnjyyeKJ6NNPw3f04IPhqC66bJdd4Lnnwuc89RRkZRUu3203uOWWUP7vf4eju6JJ6pxzQvmHH8KaNYXLd9sNunYN5UuWFByl5s/TsGHYrhC+71q1Co5iYymrCSJ6J7x5c8E6bN8etk/0jjI3N+xkO3cOn/3oo8WPmE8/PSTyn34Kn1N0R33TTaHGOm9eOIiJ3glD2KaXXBKOuI8+uvj6jB8fvr9Fi+Caawqmm4Wdba9eIcb168PBR716BTviPfYoqNW2aQN9+xbeSef3H0KotbdoUXxH3qxZKL/mmlDrjn5vWlrB8ocPD4+StG0bu1WjTZuS31NeNb5pSKqmeJrstm8PPxazsJPbsCHUbLZuLfi7zz7hh5WTA8uWFS8/5hho1CgcvX32WeGyvLzwA91lF5g2LVTFi5bfe2/48b70UtjRR5fl5RX0Gd13H7z6auHyevXCDhJCTWfChILyrVthzz0Ljup694a33ir8XXToEHZiEJropk0rXH744SG5ABx2WEFfVL7jjitoXujQoaAvKz/ZnXlm2FlC6Mv66qtwFBprm6xfD2vXFu5Tu/LKsIPPywvLK+rGG8P3sm5dSBj5n52/sxw+PCTnn34KO+XoHWX9+qEG2bt3qI3edVfxHW2vXnDwwaFfLf8kj+j3779/+NzNm0P8+dOjd8LVQUU1o5bWNKREIEmhPoLC8tuFoxORGbRqFcqXLg07s+jyhg2hW7dQPmVK2CFGJ6IWLcKOFML3umpV4fIDDoBBg0L59deHGl0sZgVHrNE7406dQo0NQkIsesTcokVIdu6h/6hevZAwqtNOuKpI9FlDSgSSNFW9PTrVVOSJFVL1lJYIUrxrTJJpwICwg9m+PfxVEkiukSOLX/OyoydWSPWiRCAiQEjEY8aEGoBZ+JuqTXWpJqGJwMx6mdkiM1tqZsNilNczs1ci5TPNLD2R8YhI6VRLS00JSwRmVhsYBZwGHARcYGYHFZntMmCNu+8PPADcm6h4REQktkTWCLoBS919mbtvAV4G+haZpy8QOdOa8cCJZjqnQESkMiUyEbQCVka9zo5MizmPu+cB64CmRRdkZkPMLMvMsnJychIUrohIakpkIoh1ZF/0XNV45sHdx7h7hrtnNG/evEKCExGRIJGJIBvYJ+p1a+D7kuYxszRgV+CXBMYkIiJFJHKsoVlAezNrB3wH9AcuLDLPROASYDpwLvCel3GF2+zZs1eZWYzLXuLSDFhV5lzVg9alaqop61JT1gO0LvnallSQsETg7nlmdjUwGagNPO3u88xsBJDl7hOBp4AXzGwpoSbQP47l7nDbkJlllXRlXXWjdamaasq61JT1AK1LPBI6+qi7TwImFZl2W9TzXOC8RMYgIiKl05XFIiIpLtUSwZhkB1CBtC5VU01Zl5qyHqB1KVO1G31UREQqVqrVCEREpAglAhGRFFcjE4GZPW1mP5vZVyWUm5k9HBn1dK6ZHVbZMcYjjvXoaWbrzGxO5HFbrPmqAjPbx8zeN7MFZjbPzIrdgbg6bJc416NabBczq29mn5rZF5F1uSPGPNVihOA412WQmeVEbZfByYg1HmZW28w+N7M3Y5RV/DZx9xr3AI4FDgO+KqH8dOA/hCEuugMzkx3zDq5HT+DNZMcZ57q0BA6LPG8MLAYOqm7bJc71qBbbJfI9N4o8rwPMBLoXmecqYHTkeX/glWTHvRPrMgj4Z7JjjXN9/gy8GOv/KBHbpEbWCNx9GqUPVdEXeN6DGcBuZtaycqKLXxzrUW24+w/u/lnk+QZgAcUHIazy2yXO9agWIt/zr5GXdSKPomePVIsRguNcl2rBzFoDZwBPljBLhW+TGpkI4hDPyKjVxVGR6vB/zOzgZAcTj0hVtivhqC1atdoupawHVJPtEmmCmAP8DLzr7iVuEy9lhOCqII51ATgn0uw43sz2iVFeFTwI3ARsL6G8wrdJqiaCuEY9rQY+A9q6e2fgEeC1JMdTJjNrBEwArnP39UWLY7ylSm6XMtaj2mwXd9/m7l0Ig0J2M7NDisxSbbZJHOvyBpDu7ocC/6XgqLrKMLPewM/uPru02WJM26ltkqqJIJ6RUas8d1+fXx32MJxHHTNrluSwSmRmdQg7z0x3/3eMWarFdilrParbdgFw97XAVKBXkaJqN0JwSevi7qvdfXPk5RPA4ZUcWjx6AH3MbDnhZl4nmNnYIvNU+DZJ1UQwEbg4cpZKd2Cdu/+Q7KDKy8z2ym8bNLNuhO25OrlRxRaJ8ylggbv/o4TZqvx2iWc9qst2MbPmZrZb5HkD4CRgYZHZ8kcIhjhHCE6GeNalSH9TH0L/TpXi7re4e2t3Tyd0BL/n7gOLzFbh2yShg84li5m9RDhzo5mZZQN/I3Qe4e6jCQPhnQ4sBTYClyYn0tLFsR7nAleaWR6wCehfFX+kET2Ai4AvI+24ALcCbaBabZd41qO6bJeWwHMW7i9eCxjn7m/aTo4QnCTxrMu1ZtYHyCOsy6CkRVtOid4mGmJCRCTFpWrTkIiIRCgRiIikOCUCEZEUp0QgIpLilAhERFKcEoFIhJltixqZco6ZDavAZadbCaPIiiRbjbyOQGQHbYoMUSCSUlQjECmDmS03s3sj491/amb7R6a3NbMpkUHMpphZm8j0Fmb2amTQuS/M7HeRRdU2syci4+W/E7kCFjO71szmR5bzcpJWU1KYEoFIgQZFmobOjypb7+7dgH8SRock8vz5yCBmmcDDkekPAx9EBp07DJgXmd4eGOXuBwNrgXMi04cBXSPLuSJRKydSEl1ZLBJhZr+6e6MY05cDJ7j7ssiAcz+6e1MzWwW0dPetkek/uHszM8sBWkcNcJY/ZPW77t4+8vpmoI6732lmbwO/EkYpfS1qXH2RSqEagUh8vITnJc0Ty+ao59so6KM7AxhFGA1zdmRESZFKo0QgEp/zo/5Ojzz/hIIBvwYAH0WeTwGuhN9ultKkpIWaWS1gH3d/n3Azkt2AYrUSkUTSkYdIgQZRI4oCvO3u+aeQ1jOzmYSDpwsi064FnjazG4EcCkZLHQqMMbPLCEf+VwIlDaddGxhrZrsSbjjyQGQ8fZFKoz4CkTJE+ggy3H1VsmMRSQQ1DYmIpDjVCEREUpxqBCIiKU6JQEQkxSkRiIikOCUCEZEUp0QgIpLi/h+qjVPkyr8i2gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5xVdb3/8debAUFEBQGVGGBAMPGKOaKlmV1UMpMung6KpR6TLtrF/JWalR6TR56TpzoePXWoLCvSzE5mpnm8ppYgg4KKpg6EMt4YuQmCyMDn98d3DbNnz5phZpg9N97Px2M/9l7r+11rf9deM+uzvt/vWuuriMDMzKxYn64ugJmZdU8OEGZmlssBwszMcjlAmJlZLgcIMzPL5QBhZma5HCCs00gqk7RO0uiOzNuVJI2X1OHXikv6gKSlBdPPSHp3a/K247t+Iunr7V3eeq++XV0A674krSuYHAhsBDZn05+JiNltWV9EbAYGdXTeHUFEvL0j1iPp08DpEXFswbo/3RHrtt7HAcKaFRFbD9DZGeqnI+Lu5vJL6hsRdZ1RNjMrPTcxWbtJukLSbyTdIGktcLqkd0qaI2m1pJclXS2pX5a/r6SQVJFN/ypLv0PSWkkPSxrb1rxZ+gclPStpjaT/kvRXSWc2U+7WlPEzkqolrZJ0dcGyZZK+L2mFpMXAlBZ+n29IurFo3rWSvpd9/rSkp7PtWZyd3Te3rhpJx2afB0r6ZVa2RcBhOd+7JFvvIkknZ/MPAq4B3p01371W8NteVrD8Z7NtXyHpFkkjWvPbtPF3btI0J+mhwn2Wfc/fs+14UtIhzX2XlUhE+OXXNl/AUuADRfOuAN4CPkw62dgZOBw4glQ7HQc8C5yX5e8LBFCRTf8KeA2oBPoBvwF+1Y68ewJrgalZ2leATcCZzWxLa8r4B2B3oAJYWb/twHnAIqAcGAo8kP6Ncr9nHLAO2KVg3cuBymz6w1keAe8DNgAHZ2kfAJYWrKsGODb7fBVwPzAEGAM8VZT3E8CIbJ+clpVhryzt08D9ReX8FXBZ9vn4rIyTgAHAfwP3tua3aePvPL74dwMeqt9nwKnAMlLwE7AvMKqr/w92tJdrELa9HoqIP0bElojYEBHzImJuRNRFxBJgFvCeFpa/OSKqImITMJt0YGpr3pOABRHxhyzt+6RgkquVZfxORKyJiKWkg3H9d30C+H5E1ETECuDKFr5nCfAkKXABHAesjoiqLP2PEbEkknuBe4DcjuginwCuiIhVEfE8qVZQ+L03RcTL2T75NSm4V7ZivQDTgZ9ExIKIeBO4CHiPpPKCPM39No2042+h0KeBKyNifvb7PBsRy1q5rHUQBwjbXo3+aSXtJ+lPkl6R9DpwOTCsheVfKfi8npY7ppvL+7bCckREkM64c7WyjK36LuD5FsoL8GvS2TCks/mtHfuSTpI0V9JKSatJZ+8t/Vb1RrRUBklnSlqYNe2sBvZr5Xohbd/W9UXE68AqYGRBnlbts3b8LRQaBSxuZV4rEQcI217Fl3j+D+mseXxE7AZ8i9REUEovk5p8AJAkGh/Qim1PGV8mHbzqbesy3N8AH8jOwKeSAgaSdgZuBr5Dav4ZDPxfK8vxSnNlkDQO+CHwOWBott6/F6x3W5fkvkRqtqpf366kpqwXW1GuYi39zm9k6x9YkH/vgs/LgH3a8Z3WgRwgrKPtCqwB3pA0EfhMJ3znbcA7JH1YUl/gS8DwEpXxJuDLkkZKGgpc2FLmiHiV1Lb+M+CZiHguS+oP7ATUApslnQS8vw1l+LqkwUr3iZxXkDaIFARqSbHy06QaRL1XgfL6zuIcNwBnSzpYUn9SAHswIpqtkbWgpd/5lex1etbxP4OCwAT8BPiapEOVTJBUGBStEzhAWEe7ADiD1Gn8P6Qz6JLKDsL/DHwPWEE683yMdN9GR5fxh6S+gieAeaRawLb8mtTp/OuCMq8Gzgd+T+roPYUU6FrjUlJNZilwB/CLgvU+DlwNPJLl2Q+YW7DsXcBzwKuSCpuK6pf/M6kp6PfZ8qNJ/RLt0ezvnDUDngN8ndRfNL6wnBFxA/Bv2TKvA/9LqslYJ1LaT2a9h6QyUlPJKRHxYFeXx6yncg3CegVJUyTtnjWLfBOoI51Fm1k7OUBYb3E0sITUXDEF+EhENNfEZGat4CYmMzPL5RqEmZnl6jUP6xs2bFhUVFR0dTHMzHqU+fPnvxYRuZeF95oAUVFRQVVVVVcXw8ysR5HU7NMA3MRkZma5HCDMzCyXA4SZmeVygDAzs1wlCxCSrpO0XNKTzaQrG2GqWtLjkt5RkHaGpOey1xmlKqOZmTWvlDWIn9PCcIzAB4EJ2WsG6SFoSNqD9DCyI4DJwKWS/JCuHcjs2VBRAX36pPfZs7e1hNmOqdT/KyULEBHxAOkplc2ZCvwiGy1qDjA4G/v2BOCuiFgZEatIT59sKdBYLzJ7NsyYAc8/DxHpfcYMB4nuwIG7e+mM/5WuvA9iJI1HxarJ5jU3v4nsGfIzAEaP3ta4LdZdbN6cXjvtlKYXL4Y330yvCy6A9esb51+/Hs4/H3bbLU1LMH487LcfvPUW3HNPw/z69wkTYNy4tOxf/9o0/e1vh/JyWLcO6m+fKU7fay94/XV4/PGG+fWvt78d9tgDVq+GZ55puvy++8Luu8OqVfCPf+SXb5ddYOVKqKlpuv599oEBA1L6q682XX7cOOjXL6WvXNk0fcwYKCtL379mTeM0KW27lMr/xhuNl+/TB/bOhu55/fW0X373u7RvNmxI8+sPRm+8ASedlA5Q9a8+fWBk9h/78stpH9SnbdkC/funAAPpt6lP37Ilve+yS9q/AIsWpe8sTN99d5g4MaVXVaXy1adFwPDhcMABKf3++9PfSGH5ysvhoINS+m23pb/FwvR99oFDDknzf/e7xmkRad2HHJLKddNNTdMrK1P666/DDTc0TT/mmPT9tbXpYF6c/sEPpu9Ytgx+9aum6f/0T+lv/2tfy/9fueQSmN7eB7QXK+WA16RBzZ9sJu1PwNEF0/eQBij/KvCNgvnfBC7Y1ncddthhYa1TVxexfn3D9LJlEU8+GVFVFfHQQxF33x3xt781pP/2txHXXBNx1VURV1wR8Y1vRPzkJw3pX/xixCmnRJx0UsQHPhBx9NERF1/ckL7//hHDhkUMGhTRt2/6M/+Xf2lILysr/hfY9uuSS9Kyy5fnp3/nOyl98eL89GuuSekLFuSnX399Sn/wwfz0//3flH7HHfnpd92V0m+6KT/94YdT+k9/mp/+5JMp/Qc/yE9//vmU/u1v56evXJnSL7wwP33TppT++c83Tevfv2HffPKTLe+HnXduOm/s2Ibl3//+pukHH9yQPnly0/SjjmpInzixafoJJzSkl5c3TT/llIb0wYObpp91Vst/e1/8Ykpbv77lv71XX235b6+6urR/e1J+uhRtAlRF5B9Xu7IGUUPjYRPLSc/wrwGOLZp/f6kKMXt2irgvvACjR8PMmR0YfZtRVwcbN8LAgemsrbYWli9vOIvesCGlf+hDKf9f/gJPPtmQ/uab6SztX/81pV99NTz4YOP0wYPhj39M6aefDnfe2ZBWVwf775/OzgCmTWs4y643eTLMzYZvueIKWLiwIU2CE06As89O048/nso/YEDjV73jjoNNm9K8/v3T+6GHNqRff306Gx4wAM45J62r2N57p7O9iIZpSNs5Z07D/Pr3+grl294GDz3UNH2ffRre77uvaXr9GeqBB8JddzX+FwSYNCm9H3YY3H570+UPOSS9H3UU/OEPTdP33Te9v+99Tc9SIZ3lQjqb3HvvpssPHZrep05NZ+PF6QOzgTz/6Z9Sbac+rT69T9a4fNppaVsKl+9T0PD8qU/BkUfCueeS68034Uc/aqiZ9OkDu+7akH7hhWkd9WkSDCnoUZw5M9WA6tOkhm0DuPbaVMsrTB9e8FCI2bNTDaGw9rXnng3pd9yRagKF6YXLz53bOE2CYdmo2f37p/+74vT68g0dmmq/xemDB6f0MWPgxRebptf/PgcckLa9OL3+f+dd70rHguL0srKUPmpUOm4V68jGlJI+zVVSBXBbRByYk/Yh0lCJJ5I6pK+OiMlZJ/V8oP6qpkeBwyKipf4MKisro62P2qhvwyuspvXvn5ozjjoKjj467eynnoJ77218AN64MVXxhg9PB66f/7xx+ptvpj/O4cPhqqvgyisblqurS9+1di0MGpSq7t/7XtPybdmS/iA+8xmYNavwt0t/nLW1aforX0kBoPDgPGJEqp4C/Nd/wd//3jh9773TwRjSAXD16sbpQ4akgyOkJo4+fRrS+vZtaI7oaHn7ZODAtP2lDtzWvIqK1KxUbMwYWLq0s0tj0HH/K5LmR0RlblqpAoSkG0g1gWGkcXAvBfoBRMSPsoHlryF1QK8HzoqIqmzZfyENRQgwMyJ+tq3va0+AaO6Pvt6cOXDEEXDddQ1ny5AOljvvDPPmpTPN66+H73638QG2f3/42c/S2cyf/pTOMovPsL/whfS+YAFUVzc+wx4wILVl1rcT15+Bl/oA3R10Ra3OWubA3T11xP9KlwSIztaeANGnT0PVupAEjzySOoIGDUr/FG+8kYJC/QHabEfjwN07OUA0w9VmM9vRtRQgduhHbcyc2dCZV2/gwDTfzGxHt0MHiOnTUxvqmDEN1467TdXMLNnhW9OnT3dAMDPLs0PXIMzMrHkOEGZmlssBwszMcjlAmJlZLgcIMzPL5QBhZma5HCDMzCyXA4SZmeVygDAzs1wOEGZmlssBwszMcjlAmJlZrpIGCElTJD0jqVrSRTnpYyTdI+lxSfdLKi9I2yxpQfa6tZTlNDOzpkr2NFdJZcC1wHFADTBP0q0R8VRBtquAX0TE9ZLeB3wH+GSWtiEiJpWqfGZm1rJS1iAmA9URsSQi3gJuBKYW5dkfuCf7fF9OupmZdZFSBoiRwLKC6ZpsXqGFwMezzx8FdpU0NJseIKlK0hxJH8n7AkkzsjxVtbW1HVl2M7MdXikDhHLmFQ+A/f+A90h6DHgP8CJQl6WNzsZJPQ34gaR9mqwsYlZEVEZE5fDhwzuw6GZmVsoR5WqAUQXT5cBLhRki4iXgYwCSBgEfj4g1BWlExBJJ9wOHAotLWF4zMytQyhrEPGCCpLGSdgKmAY2uRpI0TFJ9GS4GrsvmD5HUvz4PcBRQ2LltZmYlVrIAERF1wHnAncDTwE0RsUjS5ZJOzrIdCzwj6VlgL2BmNn8iUCVpIanz+sqiq5/MzKzEFFHcLdAzVVZWRlVVVVcXw8ysR5E0P+vvbcJ3UpuZWS4HCDMzy+UAYWZmuRwgzMwslwOEmZnlcoAwM7NcDhBmZpbLAcLMzHI5QJiZWS4HCDMzy+UAYWZmuRwgzMwslwOEmZnlcoAwM7NcDhBmZparpAFC0hRJz0iqlnRRTvoYSfdIelzS/ZLKC9LOkPRc9jqjlOU0M7OmShYgJJUB1wIfBPYHTpW0f1G2q4BfRMTBwOXAd7Jl9wAuBY4AJgOXShpSqrKamVlTpaxBTAaqI2JJRLwF3AhMLcqzP3BP9vm+gvQTgLsiYmVErALuAqaUsKxmZlaklAFiJLCsYLomm1doIfDx7PNHgV0lDW3lskiaIalKUlVtbW2HFdzMzEobIJQzr3gA7P8HvEfSY8B7gBeBulYuS0TMiojKiKgcPnz49pbXzMwK9C3humuAUQXT5cBLhRki4iXgYwCSBgEfj4g1kmqAY4uWvb+EZTUzsyKlrEHMAyZIGitpJ2AacGthBknDJNWX4WLguuzzncDxkoZkndPHZ/PMzKyTlCxAREQdcB7pwP40cFNELJJ0uaSTs2zHAs9IehbYC5iZLbsS+DYpyMwDLs/mmZlZJ1FEk6b9HqmysjKqqqq6uhhmZj2KpPkRUZmX5jupzcwslwOEmZnlcoAwM7NcDhBmZpbLAcLMzHI5QJiZWS4HCDMzy+UAYWZmuRwgzMwslwOEmZnlcoAwM7NcDhBmZpbLAcLMzHI5QJiZWS4HCDMzy+UAYWZmuUoaICRNkfSMpGpJF+Wkj5Z0n6THJD0u6cRsfoWkDZIWZK8flbKcZmbWVN9SrVhSGXAtcBxQA8yTdGtEPFWQ7RukoUh/KGl/4HagIktbHBGTSlU+MzNrWSlrEJOB6ohYEhFvATcCU4vyBLBb9nl34KUSlsfMzNqglAFiJLCsYLomm1foMuB0STWk2sMXCtLGZk1Pf5H07rwvkDRDUpWkqtra2g4supmZlTJAKGdeFE2fCvw8IsqBE4FfSuoDvAyMjohDga8Av5a0W9GyRMSsiKiMiMrhw4d3cPHNzHZspQwQNcCogulymjYhnQ3cBBARDwMDgGERsTEiVmTz5wOLgX1LWFYzMytSygAxD5ggaayknYBpwK1FeV4A3g8gaSIpQNRKGp51ciNpHDABWFLCspqZWZGSXcUUEXWSzgPuBMqA6yJikaTLgaqIuBW4APixpPNJzU9nRkRIOga4XFIdsBn4bESsLFVZzcysKUUUdwv0TJWVlVFVVdXVxTAz61EkzY+Iyrw030ltZma5HCDMzCyXA4SZmeUqWSe1mVlH2bRpEzU1Nbz55ptdXZQea8CAAZSXl9OvX79WL+MAYWbdXk1NDbvuuisVFRVIeffgWksighUrVlBTU8PYsWNbvZybmMys23vzzTcZOnSog0M7SWLo0KFtroE5QJhZj+DgsH3a8/s5QJiZWS4HCDPrdWbPhooK6NMnvc+evX3rW716Nf/93//d5uVOPPFEVq9evX1f3oUcIMysV5k9G2bMgOefh4j0PmPG9gWJ5gLE5s2bW1zu9ttvZ/Dgwe3/4i7mAGFmPc6xxzZ91R+/L74Y1q9vnH/9evjSl9Ln115ruuy2XHTRRSxevJhJkyZx+OGH8973vpfTTjuNgw46CICPfOQjHHbYYRxwwAHMmjVr63IVFRW89tprLF26lIkTJ3LOOedwwAEHcPzxx7Nhw4Zmv+/HP/4xhx9+OIcccggf//jHWZ9t0JlnnsnNN9+8Nd+gQYO2fv73f/93DjroIA455BAuuqjJCM/t4gBhZr1KTU3+/BUr2r/OK6+8kn322YcFCxbw3e9+l0ceeYSZM2fy1FNpBOXrrruO+fPnU1VVxdVXX82KnC977rnnOPfcc1m0aBGDBw/md7/7XbPf97GPfYx58+axcOFCJk6cyE9/+tMWy3fHHXdwyy23MHfuXBYuXMjXvva19m9sAd8HYWY9zv33N582enRqVio2Zkx6Hzas5eVbY/LkyY3uJ7j66qv5/e9/D8CyZct47rnnGDp0aKNlxo4dy6RJkwA47LDDWLp0abPrf/LJJ/nGN77B6tWrWbduHSeccEKL5bn77rs566yzGDhwIAB77LFHezarCdcgzKxXmTkTsuPkVgMHpvkdZZdddtn6+f777+fuu+/m4YcfZuHChRx66KG59xv0799/6+eysjLq6uqaXf+ZZ57JNddcwxNPPMGll166dX19+/Zly5YtQLr57a233tr6uRSXAbcqQEg6UtKuBdO7Sjqiw0tjZradpk+HWbNSjUFK77Nmpfntteuuu7J27drctDVr1jBkyBAGDhzI3//+d+bMmdP+L8qsXbuWESNGsGnTJmYX9K5XVFQwf/58AP7whz+wadMmAI4//niuu+66rX0VK1d2zPA5ra1B/BBYVzD9RjavRZKmSHpGUrWkJr0mkkZLuk/SY5Iel3RiQdrF2XLPSGq5fmVmVmD6dFi6FLZsSe/bExwAhg4dylFHHcWBBx7IV7/61UZpU6ZMoa6ujoMPPphvfvObHHnkkdv3ZcC3v/1tjjjiCI477jj222+/rfPPOecc/vKXvzB58mTmzp27tSYzZcoUTj75ZCorK5k0aRJXXXXVdpcBWjlgkKQFETGpaN7jEXFwC8uUAc8Cx5HGp54HnBoRTxXkmQU8FhE/lLQ/cHtEVGSfbwAmA28D7gb2jYhmrynzgEFmvdfTTz/NxIkTu7oYPV7e79gRAwYtkfRFSf2y15fY9hjRk4HqiFgSEW8BNwJTi/IEsFv2eXfgpezzVODGiNgYEf8AqrP1mZlZJ2ltgPgs8C7gRVJt4AhgxjaWGQksK5iuyeYVugw4XVINcDvwhTYsi6QZkqokVdXW1rZuS8zMuolzzz2XSZMmNXr97Gc/6+pibdWqy1wjYjkwrY3rzutSL27POhX4eUT8h6R3Ar+UdGArlyUiZgGzIDUxtbF8ZmZd6tprr+3qIrSoVQFC0s/IP0D/SwuL1QCjCqbLaWhCqnc2MCVb18OSBgDDWrmsmZmVUGubmG4D/pS97iH1G6xrcYnUKT1B0lhJO5FqILcW5XkBeD+ApInAAKA2yzdNUn9JY4EJwCOtLKuZmXWA1jYxNbonXNINpCuLWlqmTtJ5wJ1AGXBdRCySdDlQFRG3AhcAP5Z0PqmGcmaky6oWSboJeAqoA85t6QomMzPreO191MYEYPS2MkXE7aTO58J53yr4/BRwVDPLzgQ68N5HMzNri9beSb1W0uvZaw3wR6BjngZlZtbROnpAiDYqfMpqT9baJqZdJe1BqjkMqJ9dslKZmbVX/YAQ9c/8rh8QArb/luodTGuvYvo08CXS1UQLgCOBh4H3la5oZmY5vvxlWLCg+fQ5c2Djxsbz1q+Hs8+GH/84f5lJk+AHP2h2lRdeeCFjxozh85//PACXXXYZknjggQdYtWoVmzZt4oorrmDq1OJ7gZtat24dU6dObbLc0qVLOemkk3jyyScBuOqqq1i3bh2XXXYZ1dXVfPazn6W2tpaysjJ++9vfss8++2zzu7ZXa69i+hJwOPB8RLwXOJR0tZGZWfdSHBy2Nb8Vpk2bxm9+85ut0zfddBNnnXUWv//973n00Ue57777uOCCC2jNo4sGDBjQ5uWmT5/Oueeey8KFC/nb3/7GiBEj2r0tbdHaTuo3I+JNSUjqHxF/l/T2kpbMzCxPC2f6QOpzaG5AiHYOBHHooYeyfPlyXnrpJWpraxkyZAgjRozg/PPP54EHHqBPnz68+OKLvPrqq+y9994trisi+PrXv95kueasXbuWF198kY9+9KNACjCdpbUBokbSYOAW4C5Jq/CNa2bWHc2c2bgPAjpkQIhTTjmFm2++mVdeeYVp06Yxe/ZsamtrmT9/Pv369aOioiJ3HIhizS1XONYDsHVdramVlEqrmpgi4qMRsToiLgO+CfwU+EgpC2Zm1i6lGBCC1Mx04403cvPNN3PKKaewZs0a9txzT/r168d9993H83m1lhzNLbfXXnuxfPlyVqxYwcaNG7ntttsA2G233SgvL+eWW24BYOPGjVvHfSi1Nt8HERF/KUVBzMw6zPTpHX7F0gEHHMDatWsZOXIkI0aMYPr06Xz4wx/eOgZD4bgNLRctf7l+/frxrW99iyOOOIKxY8c2Wt8vf/lLPvOZz/Ctb32Lfv368dvf/pZx48Z16PbladV4ED2Bx4Mw6708HkTHKNV4EGZmtoNp76M2zMysBU888QSf/OQnG83r378/c+fO7aIStZ0DhJn1CBGBlDdUTPd00EEHsaClG/o6WXu6E9zEZGbd3oABA1ixYkWXXvLZk0UEK1asaPM9FK5BmFm3V15eTk1NDR5auP0GDBhAeXl5m5ZxgDCzbq9fv36MHTu2q4uxwylpE5OkKZKekVQt6aKc9O9LWpC9npW0uiBtc0Fa8Uh0ZmZWYiWrQUgqA64FjiONMT1P0q3ZIEEARMT5Bfm/QHoIYL0NETGpVOUzM7OWlbIGMRmojoglEfEWcCPQ0rNwTwVuKGF5zMysDUoZIEYCywqma7J5TUgaA4wF7i2YPUBSlaQ5kvzcJzOzTlbKTuq8C5abu0ZtGnBzRGwumDc6Il6SNA64V9ITEbG40RdIM4AZAKNHb3OIbDMza4NS1iBqgFEF0+U0/4jwaRQ1L0XES9n7EuB+GvdP1OeZFRGVEVE5fPjwjiizmZllShkg5gETJI2VtBMpCDS5GikbeGgIaQjT+nlDJPXPPg8DjgKeKl7WzMxKp2RNTBFRJ+k84E6gDLguIhZJuhyoioj6YHEqcGM0vkVyIvA/kraQgtiVhVc/mZlZ6flx32ZmOzA/7tvMzNrMAcLMzHI5QJiZWS4HCDMzy+UAYWZmuRwgzMwslwOEmZnlcoAwM7NcDhBmZpbLAcLMzHI5QJiZWS4HCDMzy+UAYWZmuRwgzMwslwOEmZnlcoAwM7NcJQ0QkqZIekZStaSLctK/L2lB9npW0uqCtDMkPZe9zihlOc3MrKmSDTkqqQy4FjgOqAHmSbq1cOjQiDi/IP8XgEOzz3sAlwKVQADzs2VXlaq8ZmbWWClrEJOB6ohYEhFvATcCU1vIfypwQ/b5BOCuiFiZBYW7gCklLKuZmRUpZYAYCSwrmK7J5jUhaQwwFri3LctKmiGpSlJVbW1thxTazMySUgYI5cyLZvJOA26OiM1tWTYiZkVEZURUDh8+vJ3FNDOzPKUMEDXAqILpcuClZvJOo6F5qa3LmplZCZQyQMwDJkgaK2knUhC4tTiTpLcDQ4CHC2bfCRwvaYikIcDx2TwzM+skJbuKKSLqJJ1HOrCXAddFxCJJlwNVEVEfLE4FboyIKFh2paRvk4IMwOURsbJUZTUzs6ZUcFzu0SorK6Oqqqqri2Fm1qNImh8RlXlpvpPazMxyOUCYmVkuBwgzM8vlAGFmZrkcIMzMLJcDhJlZTzV7NlRUQJ8+6X327A5dfcnugzAzsxKaPRtmzID169P088+naYDp0zvkK1yDMDPrSdavhyeegPPPbwgOhWmXXNJhX+UahJlZd/PGG7B4MTz3HFRXN7xXV8OLL7a87AsvdFgxHCDMzLrCunUNB/3CAPDcc/Dyy43z7rknTJgAH/hAeh8/Hr78ZXjllabrHT26w4roAGFmVipr1zYNAvXvxQf3vfdOB/4pU9L7+PEpGOyzD+y2W9N119U17oMAGDgQZs7ssOI7QJiZbY/XX88PANXV8OqrjfOOGJEO+iee2BAA6oPBoEFt+976juhLLknNSqNHp+DQQR3U4If1mZlt25o1+QHgueegeDTLkSMb1wDqP++zT9uDQIYPZfwAAA3oSURBVCdo6WF9rkGYmQGsWpUfAKqr4bXXGuctL08H/Y98pHEwGDcOdtmla8pfAg4QZrbjWLmy+eagFSsa5x01Kh30P/axxrWBceNSW/8OoKQBQtIU4D9JAwb9JCKuzMnzCeAy0pjTCyPitGz+ZuCJLNsLEXFyKctqZr1ARAoCzTUHrVrVkFdK7fbjx8MppzRuDho3Dnbeueu2o5soWYCQVAZcCxxHGmN6nqRbI+KpgjwTgIuBoyJilaQ9C1axISImlap8ZtZDRaQmn+aag1avbsgrwZgx6aA/bVrj5qCxY2HAgK7bjh6glDWIyUB1RCwBkHQjMBV4qiDPOcC1EbEKICKWl7A8ZtZTRKTO3+aag9asacjbp08KAhMmwGmnNW4OGjsW+vfvuu3o4UoZIEYCywqma4AjivLsCyDpr6RmqMsi4s9Z2gBJVUAdcGVE3FL8BZJmADMARnfgzSFm1gkiYPny/ABQXZ0uH61XVpYeRjd+PBx5ZOPmoLFjYaedumwzerNSBgjlzCu+prYvMAE4FigHHpR0YESsBkZHxEuSxgH3SnoiIhY3WlnELGAWpMtcO3oDrIvMnl3Sa7utndqzXyLSDWHN3Sy2bl1D3rKydLAfPx6OOqpxc9CYMQ4CXaCUAaIGGFUwXQ68lJNnTkRsAv4h6RlSwJgXES8BRMQSSfcDhwKLsd4t7wmV55wDmzal5oM+fdJLSi/rHC09OfS009KjIZprDnrjjYb19O2bgsCECXDMMY2bg8aMgX79On/brFklu1FOUl/gWeD9wIvAPOC0iFhUkGcKcGpEnCFpGPAYMAnYAqyPiI3Z/IeBqYUd3MV8o1wPt2kTPPpoesxAYSdjS6R01lkfNDr71ZXf3dnf/6lPpeagYv36pVfh4x769UtXAeXdLDZmTAoS1m10yY1yEVEn6TzgTlL/wnURsUjS5UBVRNyapR0v6SlgM/DViFgh6V3A/0jaQnok+ZUtBQfrgd54A+bMgQcfTK85c5o+urjYzJmwZUvLr82bt52no151dV3zvd3p6QebNsF55zUOBqNGOQj0En7UhnWOlSvhoYdSMHjggVRbqKtLtYBDDknNDe9+d3rGfU1N0+XHjIGlSzu92N1SRHp1ZnD66EebPlcIvF96AT9qwzrfsmUNtYMHH4RFWcviTjvB5Mnw1a+mgPCud8Huuzcst3FjyZ9Q2ePV97/06cTxvv7jP7xfdkAOELb9IuCZZxpqBw8+mDoxAXbdNQWB005LAeHww1u+OakTnlBp7eD9skNyE5O1XV0dLFjQUDt46KGGJ1ruuWcKBPWvgw92e7RZN+YmJts+GzbAI4801A4efrjh+vWxY+GDH0zB4JhjUielLz816xUcIKyp1avhr39tqCHMm5euVpHgwAPTJY/1NYSRI7u6tGZWIg4Qlm5yKuw/eOKJ1K/Qty9UVqYri9797nR365AhXV1aM+skDhA7moh0d2vhFUaLsxvUd9kF3vlOuOyyFBCOOGKHee69mTXlANHbbd6cagT1tYOHHmoYLH3oUDj6aPjc51L/waRJftSBmW3lANHbbNyY+gzqawd//WvDUzFHj4b3v7+h/2C//Tr3Wnoz61EcIHq6119PVxXV1xAeeSQFCYCJE9MgKfV3KfuR6GbWBg4QPc3y5Y37DxYsSI9CKCuDd7wDzj03BYOjj4Zhw7q6tGbWgzlAdGcR6Tk39bWDBx+EZ59NaQMGpIFTLrkk1RCOPBIGDerS4ppZ7+IA0Z1s2ZKeWVRYQ3jxxZQ2eHCqFZx9dqohHHaYB1Axs5JygOhKb70F8+c37lBetSqlve1tDXcnv/vdcMAB7lA2s07lANGZ1q1rOgbChg0pbd994WMfa7jCaOxYP7LCzLqUA0QprViR7juo70N49NF0X0KfPmkMhHPOaQgIe+3V1aU1M2ukpAEiG1L0P0kjyv0kIq7MyfMJ4DIggIURcVo2/wzgG1m2KyLi+lKWtUO88ELj/oOnskHw+vdPYyBceGHDGAi77da1ZTUz24aSBQhJZcC1wHFADTBP0q2FQ4dKmgBcDBwVEask7ZnN3wO4FKgkBY752bKrSlXeNouAp59uHBBeeCGl7bZbCgKnn54CQmVly2MgmJl1Q6WsQUwGqiNiCYCkG4GpQOHY0ucA19Yf+COiflT0E4C7ImJltuxdwBTghhKWt2V1dfDYY43HQHjttZS2114pEFxwQcMYCGVlXVZUM7OOUMoAMRJYVjBdAxxRlGdfAEl/JTVDXRYRf25m2SbPlZY0A5gBMLq9dwnPnp0/Stb69TB3bkNAePhheOONtMy4cfChDzVcZTR+vDuUzazXKWWAyDtiFg9f1xeYABwLlAMPSjqwlcsSEbOAWZBGlGtzCWfPbjzO7vPPw5lnpqeZPv98wxgIBx2U5td3KL/tbW3+KjOznqaUAaIGGFUwXQ68lJNnTkRsAv4h6RlSwKghBY3CZe/v8BJecknjQdghNSW98EIaA+GYY1JfgsdAMLMdUCnvvJoHTJA0VtJOwDTg1qI8twDvBZA0jNTktAS4Ezhe0hBJQ4Djs3kdq75TudimTfBv/5aakRwczGwHVbIAERF1wHmkA/vTwE0RsUjS5ZJOzrLdCayQ9BRwH/DViFiRdU5/mxRk5gGX13dYd6jm+i381FMzMxTR9qb77qiysjKqqqratlBxHwSkEdRmzUod1WZmvZyk+RFRmZe2Yz/cZ/r0FAzGjEmd0WPGODiYmWX8qI3p0x0QzMxy7Ng1CDMza5YDhJmZ5XKAMDOzXA4QZmaWywHCzMxy9Zr7ICTVAs9vxyqGAa91UHG6Um/ZDvC2dFe9ZVt6y3bA9m3LmIgYnpfQawLE9pJU1dzNIj1Jb9kO8LZ0V71lW3rLdkDptsVNTGZmlssBwszMcjlANJjV1QXoIL1lO8Db0l31lm3pLdsBJdoW90GYmVku1yDMzCyXA4SZmeXaoQKEpOskLZf0ZDPpknS1pGpJj0t6R2eXsbVasS3HSlojaUH2+lZnl7E1JI2SdJ+kpyUtkvSlnDw9Yr+0clu6/X6RNEDSI5IWZtvxrzl5+kv6TbZP5kqq6PySblsrt+VMSbUF++TTXVHW1pJUJukxSbflpHXsfomIHeYFHAO8A3iymfQTgTsAAUcCc7u6zNuxLccCt3V1OVuxHSOAd2SfdwWeBfbvifulldvS7fdL9jsPyj73A+YCRxbl+Tzwo+zzNOA3XV3u7diWM4FrurqsbdimrwC/zvs76uj9skPVICLiAaCloUunAr+IZA4wWNKIzild27RiW3qEiHg5Ih7NPq8lDU87sihbj9gvrdyWbi/7nddlk/2yV/HVLFOB67PPNwPvl6ROKmKrtXJbegxJ5cCHgJ80k6VD98sOFSBaYSSwrGC6hh74D17gnVnV+g5JB3R1YbYlqw4fSjrLK9Tj9ksL2wI9YL9kzRgLgOXAXRHR7D6JNP78GmBo55aydVqxLQAfz5ovb5Y0qpOL2BY/AL4GbGkmvUP3iwNEY3mRtqeebTxKesbKIcB/Abd0cXlaJGkQ8DvgyxHxenFyziLddr9sY1t6xH6JiM0RMQkoByZLOrAoS4/ZJ63Ylj8CFRFxMHA3DWfg3Yqkk4DlETG/pWw589q9XxwgGqsBCs8eyoGXuqgs2yUiXq+vWkfE7UA/ScO6uFi5JPUjHVBnR8T/5mTpMftlW9vSk/YLQESsBu4HphQlbd0nkvoCu9PNmzyb25aIWBERG7PJHwOHdXLRWuso4GRJS4EbgfdJ+lVRng7dLw4Qjd0KfCq7auZIYE1EvNzVhWoPSXvXtz1Kmkza1yu6tlRNZWX8KfB0RHyvmWw9Yr+0Zlt6wn6RNFzS4OzzzsAHgL8XZbsVOCP7fApwb2Q9o91Ja7alqD/rZFLfUbcTERdHRHlEVJA6oO+NiNOLsnXofunb3gV7Ikk3kK4iGSapBriU1GlFRPwIuJ10xUw1sB44q2tKum2t2JZTgM9JqgM2ANO64z8w6azok8ATWTsxwNeB0dDj9ktrtqUn7JcRwPWSykgB7KaIuE3S5UBVRNxKCoS/lFRNOkOd1nXFbVFrtuWLkk4G6kjbcmaXlbYdSrlf/KgNMzPL5SYmMzPL5QBhZma5HCDMzCyXA4SZmeVygDAzs1wOEGbbIGlzwZM+F0i6qAPXXaFmnshr1tV2qPsgzNppQ/aoBrMdimsQZu0kaamkf8vGG3hE0vhs/hhJ92QPf7tH0uhs/l6Sfp89qG+hpHdlqyqT9ONsvIL/y+74RdIXJT2VrefGLtpM24E5QJht285FTUz/XJD2ekRMBq4hPWmT7PMvsoe/zQauzuZfDfwle1DfO4BF2fwJwLURcQCwGvh4Nv8i4NBsPZ8t1caZNcd3Upttg6R1ETEoZ/5S4H0RsSR7SN8rETFU0mvAiIjYlM1/OSKGSaoFygseDFf/WPC7ImJCNn0h0C8irpD0Z2Ad6YmvtxSMa2DWKVyDMNs+0czn5vLk2VjweTMNfYMfAq4lPV10fvZ0TrNO4wBhtn3+ueD94ezz32h4SNp04KHs8z3A52DrIDa7NbdSSX2AURFxH2mAmMFAk1qMWSn5jMRs23YueDorwJ8jov5S1/6S5pJOtk7N5n0RuE7SV4FaGp4++yVglqSzSTWFzwHNPba8DPiVpN1Jg8B8PxvPwKzTuA/CrJ2yPojKiHitq8tiVgpuYjIzs1yuQZiZWS7XIMzMLJcDhJmZ5XKAMDOzXA4QZmaWywHCzMxy/X9cAENlkfdqbwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_metric(dfhistory, metric):\n",
    "    train_metrics = dfhistory[metric]\n",
    "    val_metrics = dfhistory['val_'+metric]\n",
    "    epochs = range(1, len(train_metrics) + 1)\n",
    "    plt.plot(epochs, train_metrics, 'bo--')\n",
    "    plt.plot(epochs, val_metrics, 'ro-')\n",
    "    plt.title('Training and validation '+ metric)\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(metric)\n",
    "    plt.legend([\"train_\"+metric, 'val_'+metric])\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# 观察损失和准确率的变化\n",
    "plot_metric(dfhistory,\"loss\")\n",
    "plot_metric(dfhistory,\"auc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:23:51.292937Z",
     "start_time": "2020-10-07T02:23:51.274986Z"
    }
   },
   "outputs": [],
   "source": [
    "# 预测\n",
    "y_pred_probs = net(torch.tensor(test_set.values).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:25:26.811221Z",
     "start_time": "2020-10-07T02:25:26.797217Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:26:41.698924Z",
     "start_time": "2020-10-07T02:26:41.684922Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的保存与使用\n",
    "torch.save(net.state_dict(), './model/net_parameter.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:28:34.009767Z",
     "start_time": "2020-10-07T02:28:33.986828Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net_clone = DeepCrossing(feature_info, hidden_units)\n",
    "net_clone.load_state_dict(torch.load('./model/net_parameter.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:28:54.051097Z",
     "start_time": "2020-10-07T02:28:54.036173Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred_probs = net_clone(torch.tensor(test_set.values).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-07T02:28:58.983340Z",
     "start_time": "2020-10-07T02:28:58.973324Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
